/*-
 * Copyright (c) 2006, 2007 FTS Team
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the FTS Team nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      PlaintextIndexerWorker.cpp
 *
 * $FTS$
 */

// C Includes
#include <sysexits.h>

#include <stdio.h>
#include <sys/time.h>
#include <sys/stat.h>
#ifdef linux
#include <time.h>
#endif
#include <fcntl.h>
#include <unistd.h>

// Local Includes
#include <DocumentBuilder.hpp>
#include <DocumentInfo.hpp>
#include <FileDocumentStorage.hpp>
#include <FTSSException.hpp>
#include <FSDumper.hpp>
#include <MemoryDocumentInfoStorage.hpp>
#include <MemoryInvIndexStorage.hpp>
#include <Indexer.hpp>
#include <LemCache.hpp>
#include <PlaintextParser.hpp>
#include <Util.hpp>
#include <InvIndexBPlusStorage.hpp>

#ifdef _NO_LEMMATIZER
#include <DefaultLemmatizerInterface.hpp>
#else
# include <TurglemLemmatizer.hpp>
#endif

using namespace FTSS; // FullTextSearchSystem


#define C_MAX_FILENAME_SIZE 8192
#define C_MAX_LINE_SIZE     32768

int main(int argc, char ** argv)
{
	if (argc != 4) { fprintf(stdout, "usage: %s file_type filelist.txt output_directory\n", argv[0]); return EX_USAGE; }

	FILE * F = fopen(argv[2], "rb");
	if (F == NULL) { fprintf(stderr, "ERROR: Cannot open file `%s' for reading.\n", argv[2]); return EX_DATAERR; }
	try
	{
		using STLW::string;

		fprintf(stdout, "Initializing.\n");
		fprintf(stdout, "Data parser..."); fflush(stdout);
		BaseParser * pParser = NULL;
		if      (strcasecmp("plaintext", argv[1]) == 0) { pParser = new PlaintextParser(); }
//		else if (strcasecmp("html",      argv[1]) == 0) { pParser = new HTMLParser(); }
//		else if (strcasecmp("xml",       argv[1]) == 0) { pParser = new XMLParser(); }
//		else if (strcasecmp("atomfeed",  argv[1]) == 0) { pParser = new HTMLParser(); }
		else { fprintf(stdout, "file_type can be only { plaintext, HTML, XML, atomfeed }\n");  return EX_CONFIG; }
		fprintf(stdout, "OK\n");

#ifdef _NO_LEMMATIZER
		fprintf(stdout, "Without lemmatizer.\n");
		DefaultLemmatizerInterface     oLemmatizer;
#else
		fprintf(stdout, "Lemmatizer..."); fflush(stdout);
#ifdef linux
		TurglemLemmatizer  oLemmatizer("/usr/share/turglem");
#else
		TurglemLemmatizer  oLemmatizer("/usr/local/share/turglem");
#endif
	
		fprintf(stdout, "OK\n");
#endif
		fprintf(stdout, "Inverted index storage (memory)..."); fflush(stdout);
		//MemoryInvIndexStorage  oDataStorage;
		InvIndexBPlusStorage oDataStorage(argv[3]);
		fprintf(stdout, "OK\n");

		fprintf(stdout, "Document properties storage..."); fflush(stdout);
		MemoryDocumentInfoStorage  oFileIndexStorage;
		DocumentInfoStorage & oDocumentStorage = oFileIndexStorage;
		fprintf(stdout, "OK\n");

		fprintf(stdout, "Lemmatizer cache..."); fflush(stdout);
		LemCache             oLemCache(oLemmatizer);
		fprintf(stdout, "OK\n");

		fprintf(stdout, "Indexer..."); fflush(stdout);
		Indexer              oIndexer(&oDataStorage);
		fprintf(stdout, "OK\n");

		fprintf(stdout, "Document storage..."); fflush(stdout);
		DocumentStorage * pDocumentStorage = new FileDocumentStorage(DirFilePath(argv[3], "DocStorIndex.ftss").c_str(),
		                                                             DirFilePath(argv[3], "DocStorData.ftss").c_str(),
		                                                             C_READ_WRITE_MODE);
		fprintf(stdout, "OK\n");

		pParser -> SetHandlers(&oIndexer, pDocumentStorage, &oLemCache);

		fprintf(stdout, "\nReady to work.\n");
		char szBuffer[C_MAX_FILENAME_SIZE + 1];
		char szBufferTMP[C_MAX_LINE_SIZE + 1];

		struct timeval sTVBegin;
		gettimeofday(&sTVBegin, NULL);
		oIndexer.StartIndexer();

		Zone oZone;
		snprintf(oZone.name, C_MAX_ZONE_NAME_LENGTH, "Default zone for plaintext data");
		oZone.zone_id    = 0;
		oZone.zone_flags = C_ZONE_TEXTZONE;
		((InvIndexStorage *)&oDataStorage) -> StoreZones(&oZone, 1);

		while (!feof(F))
		{
			if (fgets(szBuffer, C_MAX_FILENAME_SIZE, F) != NULL)
			{
				Size_t iLen = strlen(szBuffer) - 1;
				szBuffer[iLen] = '\0';

				FILE * FSRC = fopen(szBuffer, "rb");
				if (FSRC == NULL) { fprintf(stdout, "Cannot open file `%s' for reading\n", szBuffer); }
				else
				{
#ifdef _DEBUG
					fprintf(stderr, "Processing file `%s'\n", szBuffer);
#endif
					struct stat oFileStat;
					fstat(fileno(FSRC), &oFileStat);

					DocumentInfo oDocumentInfo;
					oDocumentInfo.doc_id = pParser -> StartDocument(szBuffer, iLen);

					while (!feof(FSRC))
					{
						if (fgets(szBufferTMP, C_MAX_FILENAME_SIZE, FSRC) != NULL)
						{
							pParser -> ParseChunk(szBufferTMP, strlen(szBufferTMP));
						}
					}
					fclose(FSRC);

					oDocumentInfo.size  = oFileStat.st_size;
					oDocumentInfo.ctime = oFileStat.st_ctime;
					oDocumentInfo.mtime = oFileStat.st_mtime;
					oDocumentInfo.atime = oFileStat.st_atime;
					oDocumentInfo.size_terms  = pParser -> GetTokenPos();
					oDocumentInfo.doc_core_id = pParser -> EndDocument();

//
				//	fprintf(stderr, "oDocumentInfo.EndDocument = %d\n", oDocumentInfo.doc_core_id);
					oDocumentStorage.StoreInfo(oDocumentInfo);
				}
			}
		}
		oIndexer.StopIndexer();
		struct timeval sTVEnd;
		gettimeofday(&sTVEnd, NULL);

		dynamic_cast<FileDocumentStorage *>(pDocumentStorage) -> Sync();
		delete pDocumentStorage;

		fprintf(stdout, "Indexing Done in %f msecs.\n", (1000.0 * (sTVEnd.tv_sec - sTVBegin.tv_sec) + 0.001 * (sTVEnd.tv_usec - sTVBegin.tv_usec)));

		/*int iFileHandle = open(DirFilePath(argv[3], "InvIndex.ftss").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
		if (iFileHandle != -1)
		{
			FSDumper oDumper(iFileHandle);
			if (oDataStorage.Dump(&oDumper) != 0) { fprintf(stdout, "ERROR: Cannot dump main index!\n"); }
			close(iFileHandle);
		}
		else { fprintf(stdout, "ERROR: Cannot open main index file for writing!\n"); }*/

		INT_32 iFileHandle = open(DirFilePath(argv[3], "DocumentsDB.ftss").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
		if (iFileHandle != -1)
		{
			FSDumper oDumper(iFileHandle);
			if (oFileIndexStorage.Dump(&oDumper) != 0) { fprintf(stdout, "ERROR: Cannot dump document info!\n"); }
			close(iFileHandle);
		}
		else { fprintf(stdout, "ERROR: Cannot open document info file for writing!\n"); }
	}
	catch (LogicException & e)
	{
		fprintf(stderr, "ERROR: %s!\n", e.what());
		return EX_SOFTWARE;
	}
	catch (UnixException & e)
	{
		fprintf(stderr, "ERROR: %s!\n", e.what());
		return EX_SOFTWARE;
	}
	catch(...)
	{
		fprintf(stderr, "ERROR\n");
		return EX_SOFTWARE;
	}

return EX_OK;
}
// End.
